"""
流健康监控和故障恢复服务
实现流中断检测和自动重连逻辑
集成现有的异常处理机制进行故障恢复
"""
import asyncio
import logging
from typing import Dict, List, Optional, Any, Set, Tuple
from datetime import datetime, timedelta
from enum import Enum
from dataclasses import dataclass, field
import time

from sqlalchemy.orm import Session
from core.database import SessionLocal
from core.config import get_settings
from repositories.camera_repository import CameraRepository
from services.video_processing.video_stream_service import VideoStreamService

logger = logging.getLogger(__name__)
settings = get_settings()


class HealthStatus(Enum):
    """健康状态枚举"""
    HEALTHY = "healthy"
    DEGRADED = "degraded"
    UNHEALTHY = "unhealthy"
    CRITICAL = "critical"
    UNKNOWN = "unknown"


class RecoveryAction(Enum):
    """恢复动作枚举"""
    RESTART_STREAM = "restart_stream"
    RECONNECT = "reconnect"
    RESET_EXTRACTOR = "reset_extractor"
    FALLBACK_MODE = "fallback_mode"
    DISABLE_CAMERA = "disable_camera"


@dataclass
class HealthMetrics:
    """健康指标"""
    camera_id: str
    status: HealthStatus = HealthStatus.UNKNOWN
    last_frame_time: Optional[datetime] = None
    frames_received: int = 0
    frames_dropped: int = 0
    reconnection_count: int = 0
    error_count: int = 0
    last_error: Optional[str] = None
    uptime: float = 0.0
    latency: float = 0.0
    bandwidth_usage: float = 0.0
    created_at: datetime = field(default_factory=datetime.now)
    updated_at: datetime = field(default_factory=datetime.now)


@dataclass
class RecoveryAttempt:
    """恢复尝试记录"""
    camera_id: str
    action: RecoveryAction
    timestamp: datetime
    success: bool
    error_message: Optional[str] = None
    duration: float = 0.0


class StreamHealthMonitor:
    """流健康监控器"""
    
    def __init__(self):
        self.is_running = False
        self.health_metrics: Dict[str, HealthMetrics] = {}
        self.recovery_history: Dict[str, List[RecoveryAttempt]] = {}
        self.monitored_cameras: Set[str] = set()
        
        # 监控配置
        self.check_interval = settings.STREAM_HEALTH_CHECK_INTERVAL
        self.frame_timeout = settings.STREAM_TIMEOUT
        self.max_recovery_attempts = getattr(settings, 'MAX_RECOVERY_ATTEMPTS', 3)
        self.recovery_cooldown = getattr(settings, 'RECOVERY_COOLDOWN', 60)
        
        # 服务依赖
        self.video_service = VideoStreamService()
        
        # 监控任务
        self._monitor_task: Optional[asyncio.Task] = None
        self._recovery_task: Optional[asyncio.Task] = None
        self._cleanup_task: Optional[asyncio.Task] = None
        
        # 统计信息
        self.monitor_stats = {
            "total_cameras_monitored": 0,
            "healthy_cameras": 0,
            "unhealthy_cameras": 0,
            "total_recovery_attempts": 0,
            "successful_recoveries": 0,
            "failed_recoveries": 0
        }
    
    async def start(self):
        """启动健康监控"""
        if self.is_running:
            logger.warning("流健康监控已在运行")
            return
        
        try:
            logger.info("启动流健康监控服务")
            
            self.is_running = True
            
            # 启动监控任务
            self._monitor_task = asyncio.create_task(self._monitoring_loop())
            self._recovery_task = asyncio.create_task(self._recovery_loop())
            self._cleanup_task = asyncio.create_task(self._cleanup_loop())
            
            logger.info("流健康监控服务启动成功")
            
        except Exception as e:
            logger.error(f"启动流健康监控服务失败: {e}")
            self.is_running = False
            raise
    
    async def stop(self):
        """停止健康监控"""
        if not self.is_running:
            return
        
        try:
            logger.info("停止流健康监控服务")
            
            self.is_running = False
            
            # 停止监控任务
            for task in [self._monitor_task, self._recovery_task, self._cleanup_task]:
                if task and not task.done():
                    task.cancel()
                    try:
                        await task
                    except asyncio.CancelledError:
                        pass
            
            logger.info("流健康监控服务已停止")
            
        except Exception as e:
            logger.error(f"停止流健康监控服务失败: {e}")
    
    async def add_camera_monitoring(self, camera_id: str) -> bool:
        """添加摄像头监控"""
        try:
            if camera_id in self.monitored_cameras:
                logger.warning(f"摄像头 {camera_id} 已在监控中")
                return True
            
            # 创建健康指标
            metrics = HealthMetrics(camera_id=camera_id)
            self.health_metrics[camera_id] = metrics
            self.recovery_history[camera_id] = []
            
            # 添加到监控列表
            self.monitored_cameras.add(camera_id)
            
            # 更新统计
            self.monitor_stats["total_cameras_monitored"] = len(self.monitored_cameras)
            
            logger.info(f"添加摄像头 {camera_id} 到健康监控")
            return True
            
        except Exception as e:
            logger.error(f"添加摄像头 {camera_id} 监控失败: {e}")
            return False
    
    async def remove_camera_monitoring(self, camera_id: str) -> bool:
        """移除摄像头监控"""
        try:
            if camera_id not in self.monitored_cameras:
                logger.warning(f"摄像头 {camera_id} 不在监控中")
                return True
            
            # 从监控列表移除
            self.monitored_cameras.discard(camera_id)
            
            # 清理数据
            if camera_id in self.health_metrics:
                del self.health_metrics[camera_id]
            if camera_id in self.recovery_history:
                del self.recovery_history[camera_id]
            
            # 更新统计
            self.monitor_stats["total_cameras_monitored"] = len(self.monitored_cameras)
            
            logger.info(f"从健康监控中移除摄像头 {camera_id}")
            return True
            
        except Exception as e:
            logger.error(f"移除摄像头 {camera_id} 监控失败: {e}")
            return False  
  
    async def get_camera_health(self, camera_id: str) -> Optional[HealthMetrics]:
        """获取摄像头健康状态"""
        try:
            if camera_id not in self.health_metrics:
                return None
            
            return self.health_metrics[camera_id]
            
        except Exception as e:
            logger.error(f"获取摄像头 {camera_id} 健康状态失败: {e}")
            return None
    
    async def get_all_health_status(self) -> Dict[str, HealthMetrics]:
        """获取所有摄像头健康状态"""
        try:
            return self.health_metrics.copy()
        except Exception as e:
            logger.error(f"获取所有健康状态失败: {e}")
            return {}
    
    async def force_recovery(self, camera_id: str, action: RecoveryAction) -> bool:
        """强制执行恢复动作"""
        try:
            if camera_id not in self.monitored_cameras:
                logger.warning(f"摄像头 {camera_id} 不在监控中")
                return False
            
            logger.info(f"强制执行摄像头 {camera_id} 恢复动作: {action}")
            
            success = await self._execute_recovery_action(camera_id, action)
            
            # 记录恢复尝试
            attempt = RecoveryAttempt(
                camera_id=camera_id,
                action=action,
                timestamp=datetime.now(),
                success=success
            )
            
            if camera_id not in self.recovery_history:
                self.recovery_history[camera_id] = []
            self.recovery_history[camera_id].append(attempt)
            
            return success
            
        except Exception as e:
            logger.error(f"强制恢复摄像头 {camera_id} 失败: {e}")
            return False
    
    async def _monitoring_loop(self):
        """监控循环"""
        try:
            logger.info("启动流健康监控循环")
            
            while self.is_running:
                try:
                    # 检查所有监控的摄像头
                    for camera_id in list(self.monitored_cameras):
                        await self._check_camera_health(camera_id)
                    
                    # 更新统计信息
                    await self._update_monitor_stats()
                    
                    # 等待下次检查
                    await asyncio.sleep(self.check_interval)
                    
                except Exception as e:
                    logger.error(f"监控循环异常: {e}")
                    await asyncio.sleep(10)
                    
        except asyncio.CancelledError:
            logger.info("监控循环被取消")
        except Exception as e:
            logger.error(f"监控循环异常退出: {e}")
    
    async def _recovery_loop(self):
        """恢复循环"""
        try:
            logger.info("启动流恢复循环")
            
            while self.is_running:
                try:
                    # 检查需要恢复的摄像头
                    cameras_to_recover = await self._identify_cameras_for_recovery()
                    
                    # 执行恢复操作
                    for camera_id, action in cameras_to_recover:
                        await self._attempt_recovery(camera_id, action)
                    
                    # 等待下次恢复检查
                    await asyncio.sleep(30)  # 每30秒检查一次恢复需求
                    
                except Exception as e:
                    logger.error(f"恢复循环异常: {e}")
                    await asyncio.sleep(15)
                    
        except asyncio.CancelledError:
            logger.info("恢复循环被取消")
        except Exception as e:
            logger.error(f"恢复循环异常退出: {e}")
    
    async def _cleanup_loop(self):
        """清理循环"""
        try:
            logger.info("启动健康监控清理循环")
            
            while self.is_running:
                try:
                    # 清理过期的恢复历史
                    await self._cleanup_recovery_history()
                    
                    # 清理不活跃的摄像头监控
                    await self._cleanup_inactive_monitoring()
                    
                    # 等待下次清理
                    await asyncio.sleep(300)  # 每5分钟清理一次
                    
                except Exception as e:
                    logger.error(f"清理循环异常: {e}")
                    await asyncio.sleep(60)
                    
        except asyncio.CancelledError:
            logger.info("清理循环被取消")
        except Exception as e:
            logger.error(f"清理循环异常退出: {e}")
    
    async def _check_camera_health(self, camera_id: str):
        """检查单个摄像头健康状态"""
        try:
            if camera_id not in self.health_metrics:
                return
            
            metrics = self.health_metrics[camera_id]
            current_time = datetime.now()
            
            # 获取视频流状态
            stream_status = await self.video_service.get_real_time_stream_status(camera_id)
            
            # 更新基本指标
            metrics.updated_at = current_time
            
            if stream_status.get("stream_health", False):
                # 流健康
                metrics.last_frame_time = current_time
                metrics.frames_received += 1
                
                # 更新延迟信息
                if "processing_time" in stream_status:
                    metrics.latency = stream_status["processing_time"]
                
                # 计算运行时间
                if "uptime_seconds" in stream_status:
                    metrics.uptime = stream_status["uptime_seconds"]
                
                # 确定健康状态
                if metrics.error_count == 0:
                    metrics.status = HealthStatus.HEALTHY
                elif metrics.error_count < 5:
                    metrics.status = HealthStatus.DEGRADED
                else:
                    metrics.status = HealthStatus.UNHEALTHY
                    
            else:
                # 流不健康
                metrics.error_count += 1
                metrics.frames_dropped += 1
                
                # 记录错误信息
                if "error" in stream_status:
                    metrics.last_error = stream_status["error"]
                
                # 检查帧超时
                if metrics.last_frame_time:
                    time_since_last_frame = (current_time - metrics.last_frame_time).total_seconds()
                    
                    if time_since_last_frame > self.frame_timeout:
                        metrics.status = HealthStatus.CRITICAL
                    elif time_since_last_frame > self.frame_timeout / 2:
                        metrics.status = HealthStatus.UNHEALTHY
                    else:
                        metrics.status = HealthStatus.DEGRADED
                else:
                    metrics.status = HealthStatus.CRITICAL
            
            # 更新数据库中的摄像头状态
            await self._update_camera_status_in_db(camera_id, metrics.status)
            
        except Exception as e:
            logger.error(f"检查摄像头 {camera_id} 健康状态失败: {e}")
            
            # 标记为未知状态
            if camera_id in self.health_metrics:
                self.health_metrics[camera_id].status = HealthStatus.UNKNOWN
                self.health_metrics[camera_id].error_count += 1
                self.health_metrics[camera_id].last_error = str(e)
    
    async def _update_camera_status_in_db(self, camera_id: str, status: HealthStatus):
        """更新数据库中的摄像头状态"""
        try:
            db = SessionLocal()
            try:
                camera_repo = CameraRepository(db)
                camera = camera_repo.get_by_id(int(camera_id))
                
                if camera:
                    # 映射健康状态到数据库状态
                    db_status = "online"
                    if status in [HealthStatus.UNHEALTHY, HealthStatus.CRITICAL]:
                        db_status = "offline"
                    elif status == HealthStatus.DEGRADED:
                        db_status = "warning"
                    elif status == HealthStatus.UNKNOWN:
                        db_status = "unknown"
                    
                    camera_repo.update_status(camera.id, db_status)
                    
            finally:
                db.close()
                
        except Exception as e:
            logger.error(f"更新摄像头 {camera_id} 数据库状态失败: {e}")
    
    async def _identify_cameras_for_recovery(self) -> List[Tuple[str, RecoveryAction]]:
        """识别需要恢复的摄像头"""
        cameras_to_recover = []
        
        try:
            current_time = datetime.now()
            
            for camera_id, metrics in self.health_metrics.items():
                # 检查是否需要恢复
                if metrics.status in [HealthStatus.UNHEALTHY, HealthStatus.CRITICAL]:
                    
                    # 检查恢复冷却时间
                    if await self._is_recovery_cooldown_active(camera_id):
                        continue
                    
                    # 检查最大恢复尝试次数
                    if await self._has_exceeded_max_recovery_attempts(camera_id):
                        continue
                    
                    # 确定恢复动作
                    action = await self._determine_recovery_action(camera_id, metrics)
                    
                    if action:
                        cameras_to_recover.append((camera_id, action))
            
            return cameras_to_recover
            
        except Exception as e:
            logger.error(f"识别需要恢复的摄像头失败: {e}")
            return []
    
    async def _determine_recovery_action(self, camera_id: str, metrics: HealthMetrics) -> Optional[RecoveryAction]:
        """确定恢复动作"""
        try:
            # 获取最近的恢复历史
            recent_attempts = await self._get_recent_recovery_attempts(camera_id)
            
            # 根据错误类型和历史确定动作
            if metrics.error_count < 3:
                return RecoveryAction.RECONNECT
            elif metrics.error_count < 10:
                # 检查是否已经尝试过重连
                if any(attempt.action == RecoveryAction.RECONNECT for attempt in recent_attempts):
                    return RecoveryAction.RESTART_STREAM
                else:
                    return RecoveryAction.RECONNECT
            elif metrics.error_count < 20:
                # 检查是否已经尝试过重启流
                if any(attempt.action == RecoveryAction.RESTART_STREAM for attempt in recent_attempts):
                    return RecoveryAction.RESET_EXTRACTOR
                else:
                    return RecoveryAction.RESTART_STREAM
            else:
                # 错误次数过多，考虑禁用
                return RecoveryAction.DISABLE_CAMERA
            
        except Exception as e:
            logger.error(f"确定摄像头 {camera_id} 恢复动作失败: {e}")
            return None
    
    async def _attempt_recovery(self, camera_id: str, action: RecoveryAction):
        """尝试恢复摄像头"""
        try:
            logger.info(f"尝试恢复摄像头 {camera_id}，动作: {action}")
            
            start_time = time.time()
            success = await self._execute_recovery_action(camera_id, action)
            duration = time.time() - start_time
            
            # 记录恢复尝试
            attempt = RecoveryAttempt(
                camera_id=camera_id,
                action=action,
                timestamp=datetime.now(),
                success=success,
                duration=duration
            )
            
            if camera_id not in self.recovery_history:
                self.recovery_history[camera_id] = []
            self.recovery_history[camera_id].append(attempt)
            
            # 更新统计
            self.monitor_stats["total_recovery_attempts"] += 1
            if success:
                self.monitor_stats["successful_recoveries"] += 1
                logger.info(f"摄像头 {camera_id} 恢复成功，耗时: {duration:.2f}s")
            else:
                self.monitor_stats["failed_recoveries"] += 1
                logger.warning(f"摄像头 {camera_id} 恢复失败，耗时: {duration:.2f}s")
            
        except Exception as e:
            logger.error(f"尝试恢复摄像头 {camera_id} 时发生异常: {e}")
            
            # 记录失败的恢复尝试
            attempt = RecoveryAttempt(
                camera_id=camera_id,
                action=action,
                timestamp=datetime.now(),
                success=False,
                error_message=str(e)
            )
            
            if camera_id not in self.recovery_history:
                self.recovery_history[camera_id] = []
            self.recovery_history[camera_id].append(attempt)
    
    async def _execute_recovery_action(self, camera_id: str, action: RecoveryAction) -> bool:
        """执行恢复动作"""
        try:
            if action == RecoveryAction.RECONNECT:
                return await self._reconnect_stream(camera_id)
            elif action == RecoveryAction.RESTART_STREAM:
                return await self._restart_stream(camera_id)
            elif action == RecoveryAction.RESET_EXTRACTOR:
                return await self._reset_extractor(camera_id)
            elif action == RecoveryAction.FALLBACK_MODE:
                return await self._enable_fallback_mode(camera_id)
            elif action == RecoveryAction.DISABLE_CAMERA:
                return await self._disable_camera(camera_id)
            else:
                logger.warning(f"未知的恢复动作: {action}")
                return False
                
        except Exception as e:
            logger.error(f"执行恢复动作 {action} 失败: {e}")
            return False
    
    async def _reconnect_stream(self, camera_id: str) -> bool:
        """重连流"""
        try:
            # 使用视频服务的重启功能
            db = SessionLocal()
            try:
                success = await self.video_service.restart_stream(camera_id, db)
                
                if success and camera_id in self.health_metrics:
                    # 重置错误计数
                    self.health_metrics[camera_id].reconnection_count += 1
                    self.health_metrics[camera_id].error_count = 0
                
                return success
            finally:
                db.close()
                
        except Exception as e:
            logger.error(f"重连摄像头 {camera_id} 流失败: {e}")
            return False
    
    async def _restart_stream(self, camera_id: str) -> bool:
        """重启流"""
        try:
            # 停止并重新启动流
            await self.video_service.stop_real_time_processing(camera_id)
            await asyncio.sleep(2)  # 等待停止完成
            
            db = SessionLocal()
            try:
                success = await self.video_service.start_real_time_processing(camera_id, db)
                
                if success and camera_id in self.health_metrics:
                    # 重置指标
                    metrics = self.health_metrics[camera_id]
                    metrics.error_count = 0
                    metrics.frames_dropped = 0
                    metrics.status = HealthStatus.HEALTHY
                
                return success
            finally:
                db.close()
                
        except Exception as e:
            logger.error(f"重启摄像头 {camera_id} 流失败: {e}")
            return False
    
    async def _reset_extractor(self, camera_id: str) -> bool:
        """重置帧提取器"""
        try:
            # 这里可以实现更深层的重置逻辑
            # 目前使用重启流作为替代
            return await self._restart_stream(camera_id)
            
        except Exception as e:
            logger.error(f"重置摄像头 {camera_id} 帧提取器失败: {e}")
            return False
    
    async def _enable_fallback_mode(self, camera_id: str) -> bool:
        """启用降级模式"""
        try:
            # 降级到快照模式或降低帧率
            if camera_id in self.health_metrics:
                metrics = self.health_metrics[camera_id]
                metrics.status = HealthStatus.DEGRADED
                
                # 这里可以实现降级逻辑，比如降低处理帧率
                logger.info(f"摄像头 {camera_id} 启用降级模式")
                return True
            
            return False
            
        except Exception as e:
            logger.error(f"启用摄像头 {camera_id} 降级模式失败: {e}")
            return False
    
    async def _disable_camera(self, camera_id: str) -> bool:
        """禁用摄像头"""
        try:
            # 停止流处理
            await self.video_service.stop_real_time_processing(camera_id)
            
            # 更新状态
            if camera_id in self.health_metrics:
                self.health_metrics[camera_id].status = HealthStatus.CRITICAL
            
            # 从监控中移除
            await self.remove_camera_monitoring(camera_id)
            
            logger.warning(f"摄像头 {camera_id} 已被禁用")
            return True
            
        except Exception as e:
            logger.error(f"禁用摄像头 {camera_id} 失败: {e}")
            return False
    
    async def _is_recovery_cooldown_active(self, camera_id: str) -> bool:
        """检查恢复冷却时间是否激活"""
        try:
            if camera_id not in self.recovery_history:
                return False
            
            recent_attempts = self.recovery_history[camera_id]
            if not recent_attempts:
                return False
            
            # 检查最近的恢复尝试
            last_attempt = recent_attempts[-1]
            time_since_last = (datetime.now() - last_attempt.timestamp).total_seconds()
            
            return time_since_last < self.recovery_cooldown
            
        except Exception as e:
            logger.error(f"检查摄像头 {camera_id} 恢复冷却时间失败: {e}")
            return False
    
    async def _has_exceeded_max_recovery_attempts(self, camera_id: str) -> bool:
        """检查是否超过最大恢复尝试次数"""
        try:
            if camera_id not in self.recovery_history:
                return False
            
            # 计算最近1小时内的恢复尝试次数
            one_hour_ago = datetime.now() - timedelta(hours=1)
            recent_attempts = [
                attempt for attempt in self.recovery_history[camera_id]
                if attempt.timestamp > one_hour_ago
            ]
            
            return len(recent_attempts) >= self.max_recovery_attempts
            
        except Exception as e:
            logger.error(f"检查摄像头 {camera_id} 最大恢复尝试次数失败: {e}")
            return False
    
    async def _get_recent_recovery_attempts(self, camera_id: str) -> List[RecoveryAttempt]:
        """获取最近的恢复尝试"""
        try:
            if camera_id not in self.recovery_history:
                return []
            
            # 获取最近30分钟的恢复尝试
            thirty_minutes_ago = datetime.now() - timedelta(minutes=30)
            recent_attempts = [
                attempt for attempt in self.recovery_history[camera_id]
                if attempt.timestamp > thirty_minutes_ago
            ]
            
            return recent_attempts
            
        except Exception as e:
            logger.error(f"获取摄像头 {camera_id} 最近恢复尝试失败: {e}")
            return []
    
    async def _update_monitor_stats(self):
        """更新监控统计信息"""
        try:
            healthy_count = sum(
                1 for metrics in self.health_metrics.values()
                if metrics.status == HealthStatus.HEALTHY
            )
            
            unhealthy_count = sum(
                1 for metrics in self.health_metrics.values()
                if metrics.status in [HealthStatus.UNHEALTHY, HealthStatus.CRITICAL]
            )
            
            self.monitor_stats.update({
                "healthy_cameras": healthy_count,
                "unhealthy_cameras": unhealthy_count
            })
            
        except Exception as e:
            logger.error(f"更新监控统计信息失败: {e}")
    
    async def _cleanup_recovery_history(self):
        """清理过期的恢复历史"""
        try:
            # 清理7天前的恢复历史
            seven_days_ago = datetime.now() - timedelta(days=7)
            
            for camera_id in list(self.recovery_history.keys()):
                history = self.recovery_history[camera_id]
                
                # 过滤掉过期的记录
                filtered_history = [
                    attempt for attempt in history
                    if attempt.timestamp > seven_days_ago
                ]
                
                self.recovery_history[camera_id] = filtered_history
            
            logger.debug("恢复历史清理完成")
            
        except Exception as e:
            logger.error(f"清理恢复历史失败: {e}")
    
    async def _cleanup_inactive_monitoring(self):
        """清理不活跃的摄像头监控"""
        try:
            # 获取当前活跃的摄像头列表
            active_cameras = await self.video_service.get_active_streams()
            
            # 移除不活跃的监控
            inactive_cameras = []
            for camera_id in list(self.monitored_cameras):
                if camera_id not in active_cameras:
                    # 检查是否长时间不活跃
                    if camera_id in self.health_metrics:
                        metrics = self.health_metrics[camera_id]
                        inactive_time = (datetime.now() - metrics.updated_at).total_seconds()
                        
                        if inactive_time > 3600:  # 1小时不活跃
                            inactive_cameras.append(camera_id)
            
            # 移除不活跃的摄像头
            for camera_id in inactive_cameras:
                await self.remove_camera_monitoring(camera_id)
                logger.info(f"移除不活跃的摄像头监控: {camera_id}")
            
        except Exception as e:
            logger.error(f"清理不活跃监控失败: {e}")
    
    def get_monitor_stats(self) -> Dict[str, Any]:
        """获取监控统计信息"""
        try:
            return {
                "is_running": self.is_running,
                "monitor_stats": self.monitor_stats.copy(),
                "monitored_cameras_count": len(self.monitored_cameras),
                "health_status_distribution": self._get_health_status_distribution(),
                "recovery_success_rate": self._calculate_recovery_success_rate()
            }
        except Exception as e:
            logger.error(f"获取监控统计信息失败: {e}")
            return {"error": str(e)}
    
    def _get_health_status_distribution(self) -> Dict[str, int]:
        """获取健康状态分布"""
        distribution = {status.value: 0 for status in HealthStatus}
        
        for metrics in self.health_metrics.values():
            distribution[metrics.status.value] += 1
        
        return distribution
    
    def _calculate_recovery_success_rate(self) -> float:
        """计算恢复成功率"""
        total_attempts = self.monitor_stats["total_recovery_attempts"]
        successful_recoveries = self.monitor_stats["successful_recoveries"]
        
        if total_attempts == 0:
            return 0.0
        
        return (successful_recoveries / total_attempts) * 100.0


# 全局实例
_health_monitor: Optional[StreamHealthMonitor] = None


def get_health_monitor() -> StreamHealthMonitor:
    """获取流健康监控器实例"""
    global _health_monitor
    if _health_monitor is None:
        _health_monitor = StreamHealthMonitor()
    return _health_monitor


async def start_health_monitor():
    """启动流健康监控器"""
    monitor = get_health_monitor()
    await monitor.start()


async def stop_health_monitor():
    """停止流健康监控器"""
    monitor = get_health_monitor()
    await monitor.stop()